import json
import os
import pandas as pd
from API.crypto_news import (get_aicoin_bulletin_by_last_id,
                             get_aicoin_bulletin_last_id,
                             get_jinse_bulletin_by_id,
                             get_social_metrics_list,
                             get_indicators_via_http)
from utils import df_into_db
import json


# 更新aicoin快讯数据
DATA_DIR = 'E:\\data'
SOCIAL_DATA_DIR = os.path.join(DATA_DIR, 'social_data')
print('更新aicoin快讯数据')
file_name = os.path.join(SOCIAL_DATA_DIR, 'aicoin_bulletin.csv')
if os.path.exists(file_name):
    historical_data = pd.read_csv(file_name, encoding='gb18030')
    last_update = historical_data['id'].astype(int).max()
else:
    last_update = 1
print(f'AICOIN快讯已更新到id为{last_update}的快讯')
newest_id = get_aicoin_bulletin_last_id()
_id = newest_id
save_count = 0
data = []
while _id >= last_update:
    _data = get_aicoin_bulletin_by_last_id(_id, dataframe=True)
    if _data.empty:
        pass
    data.append(_data)
    _id = _data['id'].min()
    save_count += 1
    if save_count >= 50:
        print(f'已经更新到{_id}')
        df = pd.concat(data, axis=0)
        df.drop_duplicates(inplace=True)
        if os.path.exists(file_name):
            df.to_csv(file_name, index=False, header=False, mode='a', encoding='gb18030')
        else:
            df.to_csv(file_name, index=False, encoding='gb18030')
        data = []
        save_count = 0

print('AICOIN快讯已更新完毕，最新id为%d' % historical_data['id'].max())
historical_data.rename(columns={'id': 'tid'}, inplace=True)
historical_data.fillna("", inplace=True)
historical_data['content'] = historical_data['content'].apply(lambda x: json.dumps(x))
df_into_db(historical_data[:100], db_name="social_data", table_name="aicoin_bulletin")
